# import requests
# import csv
# from bs4 import BeautifulSoup
#
# with open(r'D:/text_xinfadi1.csv', 'a', encoding='utf-8', newline='') as f:
#     csvwriter = csv.writer(f)
#     csvwriter.writerow(['品名', '最低价', '最高价', '平均价格'])
#
# url = "http://www.xinfadi.com.cn/getPriceData.html"
# headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"}
# res = requests.get(url, headers=headers)
# data = res.json()
# status_code = res.status_code
# print(f"网页响应码为：{status_code}")
#
# ru = [
#     {"name": item["prodName"], "lowPrice": item["lowPrice"], "highPrice": item["highPrice"], "avgPrice": item["avgPrice"]}
#     for item in data["list"]
# ]
#
# with open(r'D:/text_xinfadi1.csv', 'a', encoding='utf-8', newline='') as f:
#     for product in ru:
#         csvwriter = csv.writer(f)
#         csvwriter.writerow([product["name"], product["lowPrice"], product["highPrice"], product["avgPrice"]])
# print("爬虫完成")


# import requests
# import csv
# from bs4 import BeautifulSoup
# url = "http://www.xinfadi.com.cn/getPriceData.html"
# headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"}
# res = requests.get(url, headers=headers)
# data = res.json()
#
# with open(r'D:/text_xinfadi2.csv', 'a', encoding='utf-8', newline='') as f:
#     csvwriter = csv.writer(f)
#     csvwriter.writerow(['品名', '类别', '最低价', '最高价', '平均价格'])
#     for item in data["list"]:
#         name = item["prodName"]
#         low_price = item["lowPrice"]
#         high_price = item["highPrice"]
#         avg_price = item["avgPrice"]
#         category = ""
#         if "蔬菜" in name:
#             category = "蔬菜"
#         elif "水果" in name:
#             category = "水果"
#         elif "水产" in name:
#             category = "水产"
#         csvwriter.writerow([name, category, low_price, high_price, avg_price])
#
# print("爬虫完成")

# import requests
# import csv
# from bs4 import BeautifulSoup
#
# url = "http://www.xinfadi.com.cn/getPriceData.html"
# headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"}
# res = requests.get(url, headers=headers)
# data = res.json()
#
# with open(r'D:/text_xinfadi2.csv', 'a', encoding='utf-8', newline='') as f:
#     csvwriter = csv.writer(f)
#     csvwriter.writerow(['品名', '类别', '最低价', '最高价', '平均价格'])
#     for item in data["list"]:
#         name = item["prodName"]
#         low_price = item["lowPrice"]
#         high_price = item["highPrice"]
#         avg_price = item["avgPrice"]
#         category = ""
#         # 使用多个关键词判断水果类别
#         if any(keyword in name for keyword in ["苹果", "香蕉", "橙子", "水果"]):
#             category = "水果"
#         # 使用多个关键词判断水产类别
#         elif any(keyword in name for keyword in ["鱼", "虾", "蟹", "水产"]):
#             category = "水产"
#         else:
#             category = "蔬菜"
#         csvwriter.writerow([name, category, low_price, high_price, avg_price])
#
# print("爬虫完成")

# import requests
# import csv
# from bs4 import BeautifulSoup
#
# url = "http://www.xinfadi.com.cn/getPriceData.html"
# headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"}
#
# res = requests.get(url, headers=headers)
# data = res.json()
#
# with open(r'D:/text_xinfadi2.csv', 'a', encoding='utf-8', newline='') as f:
#     csvwriter = csv.writer(f)
#     csvwriter.writerow(['品名', '类别', '最低价', '最高价', '平均价格'])
#     for item in data["list"]:
#         name = item["prodName"]
#         low_price = item["lowPrice"]
#         high_price = item["highPrice"]
#         avg_price = item["avgPrice"]
#         category = ""
#         if "蔬菜" in name:
#             category = "蔬菜"
#         elif "水果" in name:
#             category = "水果"
#         elif "水产" in name:
#             category = "水产"
#         csvwriter.writerow([name, category, low_price, high_price, avg_price])
#
# print("数据保存完成。")


# import requests
# from bs4 import BeautifulSoup
# import csv
#
# response = requests.post('http://books.toscrape.com/')
# soup = BeautifulSoup(response.content, 'html.parser')
# categories = [a.text.strip() for a in soup.select('.side_categories > ul > li > ul > li > a')]
# print(categories)
#
# category_to_scrape = 'Travel'
# base_url = 'http://books.toscrape.com/catalogue/category/books/'
# page_url = f'{base_url}{category_to_scrape}/index.html'
# all_books_data = []
#
# while page_url:
#     response = requests.get(page_url)
#     soup = BeautifulSoup(response.content, 'html.parser')
#     books = soup.select('.product_pod')
#     for book in books:
#         title = book.h3.a['title']
#         rating = book.select_one('.star-rating')['class'][1]
#         price = book.select_one('.price_color').text
#         all_books_data.append({'Title': title, 'Rating': rating, 'Price': price})
#     next_page = soup.select_one('.next > a')
#     page_url = f'{base_url}{category_to_scrape}/{next_page["href"]}' if next_page else None
#
# with open(f'D:/{category_to_scrape}_books.csv', 'w', newline='', encoding='utf-8') as csvfile:
#     fieldnames = ['Title', 'Rating', 'Price']
#     writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
#     writer.writeheader()
#     for book_data in all_books_data:
#         writer.writerow(book_data)
#

















